package com.saibo.spider.gather;

import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.http.client.methods.CloseableHttpResponse;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.saibo.spider.commons.ConnectServer;
import com.saibo.spider.commons.SubmitRedisCommons;
import com.saibo.spider.pojo.SinaWeibo;
import com.saibo.spider.pojo.StringPojo;
import com.saibo.spider.utils.EncodeUtil;
import com.saibo.spider.utils.FileUtil;
import com.saibo.spider.utils.HttpRequestUtil;
import com.saibo.spider.utils.HttpResponseUtil;
import com.saibo.spider.utils.MD5Util;
import com.saibo.spider.utils.StringUtil;

public class MouldSinaSearchWeibo {

	String referer = "";

	public MouldSinaSearchWeibo() {

	}

	public static void main(String[] args) {
		MouldSinaSearchWeibo sc = new MouldSinaSearchWeibo();
		String s = sc.stitchUrl("萨德", "2017-03-05", "2017-03-05");
		System.out.println(s);
		sc.requestHTML(s);
	}

	public String requestHTML(String url) {
		CloseableHttpResponse httpResponse = HttpRequestUtil.getRequestMethod(url, requestHeader(referer));
		String html = HttpResponseUtil.getHtml(httpResponse);
		String seHtml = replaceScript(html);// 获取到渲染后的页面HTML
		String nextPageUrl = parseHtml(seHtml);
		while (true) {
			try {
				Thread.sleep(15 * 1000);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
			if (!StringUtil.isNullOrEmpty(nextPageUrl)) {
				httpResponse = HttpRequestUtil.getRequestMethod(nextPageUrl, requestHeader(url));
				html = HttpResponseUtil.getHtml(httpResponse);

				if (StringUtil.isNullOrEmpty(html)) {
					for (int i = 0; i < 3; i++) {

						httpResponse = HttpRequestUtil.getRequestMethod(nextPageUrl, requestHeader(url));
						html = HttpResponseUtil.getHtml(httpResponse);
						seHtml = replaceScript(html);// 获取到渲染后的页面HTML
						if (!StringUtil.isNullOrEmpty(seHtml)) {
							break;
						}
					}
				}
				seHtml = replaceScript(html);// 获取到渲染后的页面HTML
				if (StringUtil.isNullOrEmpty(html)) {
					System.out.println("未能获取到页面,地址:" + nextPageUrl);
				}
				url = nextPageUrl;
				nextPageUrl = parseHtml(seHtml);
				System.err.println("Next Page Url:" + nextPageUrl);
			} else {
				break;
			}
		}
		return null;
	}

	/**
	 * 拼接请求url ， 根据日期查询
	 * 
	 * @param targetWord
	 * @param starttime
	 * @param endtime
	 * @return
	 */
	public String stitchUrl(String targetWord, String starttime, String endtime) {
		String target = EncodeUtil.encodeURL(targetWord, "UTF-8");// 第一遍转码
		target = EncodeUtil.encodeURL(target, "UTF-8");// 第二遍转码

		StringBuilder stringUrl = new StringBuilder();
		stringUrl.append("http://s.weibo.com/weibo/");
		stringUrl.append(target);
		if (null != starttime && null != endtime) {
			stringUrl.append("&typeall=1&suball=1&timescope=custom:");
			stringUrl.append(starttime);
			stringUrl.append(":");
			stringUrl.append(endtime);
			stringUrl.append("&Refer=g");
		}
		return stringUrl.toString();
	}

	/**
	 * 取得到为渲染的Script标签内的html代码
	 */
	public String replaceScript(String html) {
		Document document = Jsoup.parse(html);
		List<Element> elementList = document.select("script");
		for (Element element : elementList) {
			String html11 = element.toString();
			html11 = html11.replaceAll("<script>STK && STK.pageletM && STK.pageletM.view\\(", "");
			html11 = html11.replaceAll("\\)</script>", "");
			if (html11.startsWith("{")) {
				Map<String, Object> mso = StringUtil.jsonToMap(html11);
				if (mso.get("pid").toString().equals("pl_weibo_direct")) {
					// FileUtil.writeLogToTxt(Jsoup.parse(StringUtil.jsonToMap(html11).get("html").toString()).toString(),"D:/searchWeibo.txt");
					return mso.get("html").toString();
				}
			} else {
				continue;
			}
		}
		return null;
	}

	public String parseHtml(String html) {
		Document document = Jsoup.parse(html);//格式化html代码
		//FileUtil.writeLogToTxt(document.toString(),"D:/searchWeibo.txt");
		Element div_feed_lists = document.select("div.search_feed>div>div.feed_lists").first();
		if(document.toString().contains("您可以尝试更换关键词，再次搜索。")){
			return null;
		}
		List<Element> weiboListElement = div_feed_lists.select("div.feed_lists>div.WB_cardwrap");
		List<SinaWeibo> ls = new ArrayList<SinaWeibo>();
		List<StringPojo> usercardList = new ArrayList<StringPojo>();
		for (Element element : weiboListElement) {
			StringPojo usercardPojo = new StringPojo();
			SinaWeibo sina = new SinaWeibo();
			Element midElement = element.select("div.WB_cardwrap>div[action-type=feed_list_item]").first();
			String mid = midElement.attr("mid");// 微博的mid
			Element wbFeedDetailEle = midElement.select("div>dl>div").first();
			Element content_div = wbFeedDetailEle.select("div>div.content").first();
			Element feed_content_div = content_div.select("div.content>div.feed_content").first();
			Element name_txt_div = feed_content_div.select("div.feed_content>a.name_txt").first();
			String name_str = name_txt_div.attr("nick-name");
			String user_index = name_txt_div.attr("href");
			String usercard = name_txt_div.attr("usercard");
			//Element ver = feed_content_div.select("div.feed_content>a.W_icon").first();
			String content = StringUtil.element2text(feed_content_div.select("div>p.comment_txt").first());
			// 发文日期
			Element dateEle = midElement.select("div>div.WB_feed_detail>dl>div.feed_list>div.content>div.feed_from")
					.first();
			Element datetimeE = dateEle.select("div>a.W_textb").first();

			String url = datetimeE.attr("href");
			String date = datetimeE.attr("date");
			//System.out.println(url);
			//System.out.println(date);
			String platform="";
			if (dateEle.select("div>a").size() < 2) {
				//System.err.println("空");
				platform = "无";
			} else {
				platform = StringUtil.element2text(dateEle.select("div>a").get(1));
				//System.out.println(platform);
			}

			// 阅读点赞
			Element forwardEle = midElement
					.select("div>div.feed_action>ul.feed_action_info>li>a[action-type=feed_list_forward]").first();
			String forward_num = StringUtil.element2text(forwardEle.select("a>span>em").first());
			Element commentEle = midElement
					.select("div>div.feed_action>ul.feed_action_info>li>a[action-type=feed_list_comment]").first();
			String comment_num = StringUtil.element2text(commentEle.select("a>span>em").first());
			Element likeEle = midElement
					.select("div>div.feed_action>ul.feed_action_info>li>a[action-type=feed_list_like]").first();
			String like_num = StringUtil.element2text(likeEle.select("a>span>em").first());
			
			System.out.println("微博MID:"+mid+"\t作者昵称:"+name_str+"\t用户主页地址:"+user_index+"\tusercard:"+usercard);
			usercardPojo.setUrl(usercard);
			usercardList.add(usercardPojo);
			System.out.println("博文地址:"+url+"\t发文时间:"+date+"\t发文平台:"+platform);
			System.out.println("博文内容:"+content);
			int forward_count = 0;
			int comment_count = 0;
			int like_count = 0;
			if(!StringUtil.isNullOrEmpty(forward_num))forward_count=Integer.parseInt(forward_num);
			if(!StringUtil.isNullOrEmpty(comment_num))comment_count=Integer.parseInt(comment_num);
			if(!StringUtil.isNullOrEmpty(like_num))like_count=Integer.parseInt(like_num);
			System.out.println("转发数:"+forward_count+"\t评论数:"+comment_count+"\t点赞数:"+like_count);
			sina.setRowkey(MD5Util.GET16BITMD5(filterUrl(url)));
			sina.setUrl(filterUrl(url));
			sina.setMid_(mid);
			sina.setCreate_time(date);
			sina.setContent(content);
			sina.setPlatform(platform);
			sina.setAuthor_rowkey(MD5Util.GET16BITMD5(filterUrl(user_index)));
			sina.setForward_count(forward_count+"");
			sina.setComment_count(comment_count+"");
			sina.setLike_count(like_count+"");
			sina.setCrawler_time(System.currentTimeMillis()+"");
			sina.setLast_update_time(System.currentTimeMillis()+"");
			ls.add(sina);
			
			// 查看是否是被转发的
			Elements isforward = midElement.select("div>div.WB_feed_together");
			int i = isforward.size();
			if (i > 0) {
				
				SinaWeibo weibo = new SinaWeibo();
				StringPojo usc = new StringPojo();
				System.out.println("↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓此微博被转发,内容如下↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓");
				// 转发人信息
				Element elementForw = midElement
						.select("div>div.WB_feed_together>div.WB_sonFeed>div.WB_sonFeed_pos>div.WB_feed_detail")
						.first();
				Element userElementForw = elementForw.select("div.WB_feed_detail>div.WB_face>div.face>a").first();
				Element userElementFF = userElementForw.select("a>img").first();
				String userUrl = userElementForw.attr("href");
				String usercardForw = userElementFF.attr("usercard");
				String usernameForwa = userElementFF.attr("title");
				// 获取内容
				Element wb_text_ele = elementForw.select("div.WB_feed_detail>div.WB_detail>div.WB_text>span").first();
				String text = StringUtil.element2text(wb_text_ele);
				// 获取点赞阅读
				Element forwardElem = elementForw
						.select("div.WB_feed_detail>div.WB_detail>div.WB_func>div.WB_handle>ul.clearfix>li>a[action-type=feed_list_forwardAgg]>em")
						.first();
				Element commentElem = elementForw
						.select("div.WB_feed_detail>div.WB_detail>div.WB_func>div.WB_handle>ul.clearfix>li>a[action-type=feed_list_commentAgg]>em")
						.first();
				Element likeElem = elementForw
						.select("div.WB_feed_detail>div.WB_detail>div.WB_func>div.WB_handle>ul.clearfix>li>a[action-type=feed_list_like]>span>em")
						.first();
				String forward_Str = StringUtil.element2text(forwardElem);
				String comment_Str = StringUtil.element2text(commentElem);
				String like_Str = StringUtil.element2text(likeElem);
				int forward_count_f = 0;
				int comment_count_f = 0;
				int like_count_f = 0;
				if(!StringUtil.isNullOrEmpty(forward_Str))forward_count_f=Integer.parseInt(forward_Str);
				if(!StringUtil.isNullOrEmpty(comment_Str))comment_count_f=Integer.parseInt(comment_Str);
				if(!StringUtil.isNullOrEmpty(like_Str))like_count_f=Integer.parseInt(like_Str);
				// 获取转发时间
				Element datetimeElem = elementForw.select("div.WB_feed_detail>div.WB_detail>div.WB_func>div.WB_from>a.W_textb").first();
				String forwardurl = datetimeElem.attr("href");
				String datetimeForwad = datetimeElem.attr("date");
				//System.err.println(forwardurl+"\t"+datetimeForwad);
				Elements fromPlat = elementForw.select("div.WB_feed_detail>div.WB_detail>div.WB_func>div.from>a");
				String platforms = "";
				if(fromPlat.size()>1){
					Element fromPlatf = elementForw.select("div.WB_feed_detail>div.WB_detail>div.WB_func>div.from>a").get(1);
					platforms = StringUtil.element2text(fromPlatf);
				}
				String forwardMid = elementForw.attr("mid");
				System.out.println("微博MID:"+forwardMid+"\t转发人昵称:"+usernameForwa+"\t转发人主页:"+userUrl+"\tusercard:"+usercardForw);
				System.out.println("转发地址:"+forwardurl+"\t转发时间:"+datetimeForwad+"\t发文平台:"+platforms);
				System.out.println("转发内容:"+text);
				System.out.println("转发数:"+forward_count_f+"\t评论数:"+comment_count_f+"\t点赞数:"+like_count_f);
				usc.setUrl(usercardForw);
				usercardList.add(usc);
				weibo.setRowkey(filterUrl(MD5Util.GET16BITMD5(forwardurl)));
				weibo.setUrl(filterUrl(forwardurl));
				weibo.setMid_(forwardMid);
				weibo.setCreate_time(datetimeForwad);
				weibo.setContent(text);
				weibo.setPlatform(platforms);
				weibo.setAuthor_rowkey(MD5Util.GET16BITMD5(filterUrl(userUrl)));
				weibo.setForward_count(forward_count_f+"");
				weibo.setComment_count(comment_count_f+"");
				weibo.setLike_count(like_count_f+"");
				weibo.setCrawler_time(System.currentTimeMillis()+"");
				weibo.setLast_update_time(System.currentTimeMillis()+"");
				
				ls.add(weibo);
				
			}
			System.out.println("------------------------------------------------------------------------------------------------------------------");
		}
		System.out.println(ls.size());
		SubmitRedisCommons.submitRedisData("sina_blog_info", StringUtil.objectToJson(ls));
		SubmitRedisCommons.submitRedisData("data_sina_weibo_usercard", StringUtil.objectToJson(usercardList));
		
		
	//	ConnectServer.sendPostRequest("http://192.168.1.177:8081/services/schedule/obtainentryinfo", StringUtil.objectToJson(usercardList));
		//ConnectServer.sendPostRequest("http://192.168.1.177:8081/services/schedule/obtainentryinfo", StringUtil.objectToJson(ls));
		
		
		if(document.select("div.search_feed>div.WB_cardwrap>div.W_pages>a.page").size()>0){
			//寻找下一页
			Element nextPageEles = document.select("div.search_feed>div.WB_cardwrap>div.W_pages>a.next").first();
			if(null==nextPageEles)
				return null;
			return "http://s.weibo.com"+nextPageEles.attr("href");
			
		}else{
			return null;
		}
		
	}

	public static Map<String, String> requestHeader(String referer) {
		Map<String, String> requestHeader = new HashMap<String, String>();
		requestHeader.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
		requestHeader.put("Accept-Encoding", "gzip, deflate, sdch");
		requestHeader.put("Accept-Language", "zh-CN,zh;q=0.8");
		requestHeader.put("Cache-Control", "max-age=0");
		requestHeader.put("Connection", "keep-alive");
		requestHeader.put("Cookie", FileUtil.txt2String(new File("C:/sinalogincookie.txt")));
		requestHeader.put("Host", "s.weibo.com");
		if (!StringUtil.isNullOrEmpty(referer)) {
			requestHeader.put("Referer", referer);
		}
		requestHeader.put("Upgrade-Insecure-Requests", "1");
		requestHeader.put("User-Agent",
				"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36");
		return requestHeader;
	}

	/**
	 * 
	 */
	public String filterUrl(String url) {
		if (url.contains("?")) {
			String[] surl = url.split("\\?");
			return surl[0];
		}
		return url;
	}

}
